R Notebook

What tweets got the most attention. frequency of tweets posted on a topic

library(rtweet)
library(ggplot2)
library(tidyr)
library(dplyr)
library(tidytext)
library(stringr)
library(scales)
library(readr)
library(lubridate)
library(vader)
library(topicmodels)
library(quanteda)
library(lubridate)
library(zoo)
library(plotly)

origop <- options("httr_oauth_cache")
options(httr_oauth_cache = TRUE)

api_key <- "5PgtS7ljq5ZbBoXnemU5qHe62"
api_secret <- "M44LeduQ4zoyDxQIkAFjeIJrpDhWnb5xASDvhahTlrAvOhN7fx"
access_token <- "743029724750942208-JLEp26XrjwvQ1CPJUXwvdUMLka82cgx"
access_secret <- "XRMeMBaOgQy2BC1Bd9iJARfMIyK40VKyII1ZRcf9nS0qd"
token <- create_token(
  app = "KyleResearchApp",
  consumer_key = api_key,
  consumer_secret = api_secret,
  access_token = access_token,
  access_secret = access_secret
)

media_agency_df <- get_timeline("News24", n = 3200)
media_agency_df <- media_agency_df %>% 
  mutate(rbind(rtweet::get_timeline("eNCA", n = 3200))) %>% 
  mutate(rbind(rtweet::get_timeline("ewnupdates", n = 3200))) %>% 
  mutate(rbind(rtweet::get_timeline("TimesLive", n = 3200))) %>% 
  mutate(rbind(rtweet::get_timeline("SABCNews", n = 3200)))

write_as_csv(media_agency_df, "data_in/media_agency_tweets")
media_agency_df <- read_csv("data_in/media_agency_tweets.csv")

media_agency_df <- media_agency_df[ , colSums(is.na(media_agency_df)) < nrow(media_agency_df)]

Comparing Tweet Archives

ggplot(media_agency_df, aes(x = created_at, fill = screen_name)) +
  geom_histogram(position = "identity", bins = 20, show.legend = FALSE) +
  facet_wrap(~screen_name, ncol = 1)

#tidy df and unnest
tidy_covid_media_df <- media_agency_df %>% 
  filter(str_detect(text, fixed(covid_dictionary, ignore_case = TRUE))) %>%
  filter(!str_detect(text, "^RT")) %>%
  mutate(text = gsub(" ?(f|ht)tp(s?)://(.*)[.][a-z]+", "", text)) %>%
  unnest_tokens(word, text, token = "tweets") %>% 
  filter(!word %in% stop_words$word,
         !word %in% str_remove_all(stop_words$word, "'"),
         str_detect(word, "[a-z]"),
         !(word =="sabcnews" |
             word =="enca" |
             word =="https" |
             word =="dstv" |
             word =="sabckzn" |
             word =="anc" |
             word =="south" |
             word =="africa" |
             word =="dstv403" |
             word =="t.co" |
             word =="itus" |
             word =="rt" |
             word =="amp"))

longer object length is not a multiple of shorter object lengthUsing `to_lower = TRUE` with `token = 'tweets'` may not preserve URLs.

#calculate a frequency for each person and word
frequency <- tidy_covid_media_df %>% 
  group_by(screen_name) %>% 
  count(word, sort = TRUE) %>% 
  left_join(tidy_covid_media_df %>% 
              group_by(screen_name) %>% 
              summarise(total = n())) %>%
  mutate(freq = n/total)

Joining, by = "screen_name"

vader_df <- vader_df(media_agency_df$text)
write_as_csv(vader_df, "data_in/vader_covid_df")

vader_df <- read_csv("data_in/vader_df.csv")
vader_covid_df <- read_csv("data_in/vader_covid_df.csv")
vader_df <- vader_df %>% mutate("X1" = row_number())
sentimentr_df <- read_csv("data_in/sentimentr_data.csv")
media_vader_df <- media_agency_df %>% left_join(vader_df, by = "X1")

#sentiment
ggplot(data=media_vader_df, aes(x=created_at, y=rollmean(compound, 30,  na.pad=TRUE))) +
       geom_line(color="pink", size=.5)+
  geom_smooth() +
  theme_minimal()+
  facet_wrap(~screen_name) +
  scale_y_continuous(expand = c(0,0), breaks = c(-0.4,-0.2,0,0.2,0.4)) +
  ylim(c(-0.4, 0.4))

#sentiment
ggplot(data=media_vader_df, aes(x=created_at, fill = screen_name)) +
  stat_smooth(media_vader_df %>% filter(screen_name == "News24"), mapping =  aes(y = mean(compound)))+
  stat_smooth(media_vader_df %>% filter(screen_name == "ewnupdates"), mapping =  aes(y = compound))+
  stat_smooth(media_vader_df %>% filter(screen_name == "eNCA"), mapping =  aes(y = compound))+
  stat_smooth(media_vader_df %>% filter(screen_name == "SABCNews"), mapping =  aes(y = compound))+
  stat_smooth(media_vader_df %>% filter(screen_name == "TimesLIVE"), mapping =  aes(y = compound))+
  theme_minimal()+
  scale_y_continuous(expand = c(0,0), breaks = c(-0.1, 0)) 
  

ggplot(data=media_vader_df, aes(x=created_at, colour = screen_name)) +
  geom_line(media_vader_df %>% filter(screen_name == "News24"), mapping =  aes(y = rollmean(compound, k = 30, na.pad = TRUE)))+
  geom_line(media_vader_df %>% filter(screen_name == "ewnupdates"), mapping = aes(y = rollmean(compound, k = 30, na.pad = TRUE)))+
  geom_line(media_vader_df %>% filter(screen_name == "eNCA"), mapping = aes(y = rollmean(compound, k = 30, na.pad = TRUE)))+
  geom_line(media_vader_df %>% filter(screen_name == "SABCNews"), mapping = aes(y = rollmean(compound, k = 30, na.pad = TRUE)))+
  geom_line(media_vader_df %>% filter(screen_name == "TimesLIVE"), mapping = aes(y = rollmean(compound, k = 30, na.pad = TRUE)))+
  theme_minimal()+
  scale_y_continuous(expand = c(0,0)) 

ggplot(media_vader_df %>% arrange(compound) %>% mutate(X1 = factor(X1, levels = X1)), aes(x=X1)) +
  geom_point(mapping =  aes(y = compound)) +
  theme_minimal()+
  scale_y_continuous(expand = c(0,0)) 

install.packages("ldatuning")


install.packages("devtools")
devtools::install_github("nikita-moor/ldatuning")


library("ldatuning")


library("topicmodels")
data("AssociatedPress", package="topicmodels")
dtm <- AssociatedPress[1:10, ]


result <- FindTopicsNumber(
  tidy_matrix,
  topics = seq(from = 2, to = 15, by = 1),
  metrics = c("Griffiths2004", "CaoJuan2009", "Arun2010", "Deveaud2014"),
  method = "Gibbs",
  control = list(seed = 77),
  mc.cores = 2L,
  verbose = TRUE
)


##  fit models... done.
##  calculate metrics:
##   Griffiths2004... done.
##   CaoJuan2009... done.
##   Arun2010... done.
##   Deveaud2014... done.


FindTopicsNumber_plot(result)